# -*- coding: utf-8 -*-
import os
import re
import urllib.request
from scrapy.selector import lxmlsel


# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: https://doc.scrapy.org/en/latest/topics/item-pipeline.html
class ScrapyimgPipeline(object):
    
    def process_item(self, item, spider):
        item = dict(item)
        for i in range(0,len(item["name"])):
            name = item["name"][i]
            #图片详情页地址：http://www.58pic.com/tuku/22248253.html
            imgDetailUrl = item["imgDetailUrl"][i]
            
            response = None
            try:
                #访问图片详情页，获取图片的真正地址
                opener = urllib.request.build_opener()
                opener.addheaders=[('User-Agent','Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.87 Safari/537.36')]
                urllib.request.install_opener(opener)
                response = urllib.request.urlopen(imgDetailUrl)
                
                detailContent = str(response.read(),encoding="ISO-8859-1")
                #转换为可以使用xpath的对象
                selector = lxmlsel.Selector(text=detailContent)  #etree.HTML(detailContent)
                #<div id="show-area-height" style='width: 100%;'><img src="http://img95.58pic.com/photo/00052/5113.jpg_wh860.jpg!/fw/650" class="show-area-pic" id="show-area-pic" alt="番茄" title="番茄" />
                imgUrl = selector.xpath("//img[@id='show-area-pic']/@src").extract()[0]
                print("==>获取到的图片路径：",imgUrl)
                
                #获取图片后缀
                suffixeParttern = '/\d*?(\.\w*?)_'
                suffixe = re.compile(suffixeParttern).findall(imgUrl)[0]
    
                #本地保存的位置
                localPath = "D:/py/58pic/"+name+suffixe
                print("==>本地保存路径：",localPath)
                
                #用于下载图片的url，此处是下载缩略图，无法下载到原图，要登录验证和开通会员
                urllib.request.urlretrieve(imgUrl,localPath)
            except Exception as e:
                print("出现异常：",e)
            finally:
                if response is not None:
                    response.close()
           
        return item
